#The code does keyword extraction from given text
#!/usr/bin/python

import string 

def sort_by_value(d): 
    u""" Returns the keys of dictionary d sorted by their values """ 
    items=d.items() 
    backitems=[ [v[1],v[0]] for v in items] 
    backitems.sort() 
    return [ backitems[i][1] for i in range(0,len(backitems))] 

def get_most_used_words(text, n):
    """ Returns the top n used words in a text without counting the top 100 most \
used words in English and German language"""
    
    # only print lower case words
    text =  text.lower()
    
    
    # replace seperators with spaces
    seperators = "\n\r\f\t\v.,/\\""''"
    #seperators = "\n\r.,"
    for seperator in seperators:
        #text = text.replace(seperator, " ")
        text = text.replace(seperator, " ")
    
    # remove invalid chars
    validchars = "abcdefghijklmnopqrstuvwxyz "
    
    charlist = list(text.lower())
    charlist = [char for char in charlist if char in validchars]
    text = "".join(charlist)
    
    # split up the words
    words  = text.split(" ")
    
    # remove empty list words
    words = [word for word in words if word != ""]
    
    # sort words
    words.sort()
    
    # define some blacklists for German, English language etc.
   
    blacklisten = "a about after again against all also and another any around as ask at back because become before begin between both but bycall can change child come consider could course day develop do down during each early end even eye face fact feel few find first follow for form from general get give go good govern great group hand have he head help here high hold home house how however I if in increase interest into it just keep know large last late lead leave life like line little long look make man many may mean might more most move much must nation need never new no not now number of off old on one only open or order other out over own part people person place plan play point possible present problem program public real right run same say school see seem set she should show since small so some stand state still such system take tell than that the then there these they thing think this those through time to too turn under up use very want way we well what when where which while who will with without word work world would write year you"

    
    # be picky what to include in the word list
    sorted = {}
    for word in words:
        # accept only words with lenght between 3 and 15
        if len(word) > 3 and len(word) < 15:
            # accept only words that are not in blacklists of the top 100 common 
	   if word not in blacklisten:
                    sorted[word] = sorted.get(word,0) + 1
   
    # use only the top n words
    top =  sort_by_value(sorted)[-n:]
    
    # don't use the number of occurances as sort criteria anymore
    top.sort()
        
    return " ".join(top)

def usage():
    """ This function prints the usage information"""
    import sys
    

def main():
    """ This is the main function"""
    import sys
    # check whether no commandline argument has been entered and show usage
    if len(sys.argv) == 1:
        usage()
    else:
        # read from stdin if "-" is present as first commandline argument
        if sys.argv[1] == "-":
            text = sys.stdin.read()
        else:
            # open file in any other case; should there be errors, print error message and usage
            try:    
                file = sys.argv[1]
                fd = open(file, 'r')
                text = fd.read()
                fd.close()
            except:
                print "ERROR: Input file could not be opened."
                usage()
    # get top 25 used words
    print get_most_used_words(text,10)

# run main function if called directly
if __name__=="__main__":
   main() 
 


